<!DOCTYPE HTML>
<html lang="zh-CN">


<head>
    <meta name="baidu-site-verification" content="code-KNXLvfbWBj" />
    <meta charset="utf-8">
    <meta name="keywords" content="Hive笔记, Aunean&#39;s Blog">
    <meta name="description" content="">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="renderer" content="webkit|ie-stand|ie-comp">
    <meta name="mobile-web-app-capable" content="yes">
    <meta name="format-detection" content="telephone=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
    <!-- Global site tag (gtag.js) - Google Analytics -->


    <title>Hive笔记 | Aunean&#39;s Blog</title>
    <link rel="icon" type="image/png" href="/favicon.png">

    <link rel="stylesheet" type="text/css" href="/libs/awesome/css/all.css">
    <link rel="stylesheet" type="text/css" href="/libs/materialize/materialize.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/aos/aos.css">
    <link rel="stylesheet" type="text/css" href="/libs/animate/animate.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/lightGallery/css/lightgallery.min.css">
    <link rel="stylesheet" type="text/css" href="/css/matery.css">
    <link rel="stylesheet" type="text/css" href="/css/my.css">

    <script src="/libs/jquery/jquery.min.js"></script>

<meta name="generator" content="Hexo 5.4.0">
<style>.github-emoji { position: relative; display: inline-block; width: 1.2em; min-height: 1.2em; overflow: hidden; vertical-align: top; color: transparent; }  .github-emoji > span { position: relative; z-index: 10; }  .github-emoji img, .github-emoji .fancybox { margin: 0 !important; padding: 0 !important; border: none !important; outline: none !important; text-decoration: none !important; user-select: none !important; cursor: auto !important; }  .github-emoji img { height: 1.2em !important; width: 1.2em !important; position: absolute !important; left: 50% !important; top: 50% !important; transform: translate(-50%, -50%) !important; user-select: none !important; cursor: auto !important; } .github-emoji-fallback { color: inherit; } .github-emoji-fallback img { opacity: 0 !important; }</style>
<link rel="alternate" href="/atom.xml" title="Aunean's Blog" type="application/atom+xml">
<link rel="stylesheet" href="/css/prism-tomorrow.css" type="text/css"></head>





<body>
    <header class="navbar-fixed">
    <nav id="headNav" class="bg-color nav-transparent">
        <div id="navContainer" class="nav-wrapper container">
            <div class="brand-logo">
                <a href="/" class="waves-effect waves-light">
                    
                    <img src="/medias/logo.png" class="logo-img" alt="LOGO">
                    
                    <span class="logo-span">Aunean&#39;s Blog</span>
                </a>
            </div>
            

<a href="#" data-target="mobile-nav" class="sidenav-trigger button-collapse"><i class="fas fa-bars"></i></a>
<ul class="right nav-menu">
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/" class="waves-effect waves-light">
      
      <i class="fas fa-home" style="zoom: 0.6;"></i>
      
      <span>首页</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/tags" class="waves-effect waves-light">
      
      <i class="fas fa-tags" style="zoom: 0.6;"></i>
      
      <span>标签</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/categories" class="waves-effect waves-light">
      
      <i class="fas fa-bookmark" style="zoom: 0.6;"></i>
      
      <span>分类</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/archives" class="waves-effect waves-light">
      
      <i class="fas fa-archive" style="zoom: 0.6;"></i>
      
      <span>归档</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/about" class="waves-effect waves-light">
      
      <i class="fas fa-user-circle" style="zoom: 0.6;"></i>
      
      <span>关于</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/contact" class="waves-effect waves-light">
      
      <i class="fas fa-comments" style="zoom: 0.6;"></i>
      
      <span>留言板</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/friends" class="waves-effect waves-light">
      
      <i class="fas fa-address-book" style="zoom: 0.6;"></i>
      
      <span>友情链接</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/navigate" class="waves-effect waves-light">
      
      <i class="fas fa-location-arrow" style="zoom: 0.6;"></i>
      
      <span>导航</span>
    </a>
    
  </li>
  
  <li>
    <a href="#searchModal" class="modal-trigger waves-effect waves-light">
      <i id="searchIcon" class="fas fa-search" title="搜索" style="zoom: 0.85;"></i>
    </a>
  </li>
</ul>


<div id="mobile-nav" class="side-nav sidenav">

    <div class="mobile-head bg-color">
        
        <img src="/medias/logo.png" class="logo-img circle responsive-img">
        
        <div class="logo-name">Aunean&#39;s Blog</div>
        <div class="logo-desc">
            
            Never really desperate, only the lost of the soul.
            
        </div>
    </div>

    

    <ul class="menu-list mobile-menu-list">
        
        <li class="m-nav-item">
	  
		<a href="/" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-home"></i>
			
			首页
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/tags" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-tags"></i>
			
			标签
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/categories" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-bookmark"></i>
			
			分类
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/archives" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-archive"></i>
			
			归档
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/about" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-user-circle"></i>
			
			关于
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/contact" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-comments"></i>
			
			留言板
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/friends" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-address-book"></i>
			
			友情链接
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/navigate" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-location-arrow"></i>
			
			导航
		</a>
          
        </li>
        
        
        <li><div class="divider"></div></li>
        <li>
            <a href="https://github.com/Aunean-ls" class="waves-effect waves-light" target="_blank">
                <i class="fab fa-github-square fa-fw"></i>Fork Me
            </a>
        </li>
        
    </ul>
</div>


        </div>

        
            <style>
    .nav-transparent .github-corner {
        display: none !important;
    }

    .github-corner {
        position: absolute;
        z-index: 10;
        top: 0;
        right: 0;
        border: 0;
        transform: scale(1.1);
    }

    .github-corner svg {
        color: #0f9d58;
        fill: #fff;
        height: 64px;
        width: 64px;
    }

    .github-corner:hover .octo-arm {
        animation: a 0.56s ease-in-out;
    }

    .github-corner .octo-arm {
        animation: none;
    }

    @keyframes a {
        0%,
        to {
            transform: rotate(0);
        }
        20%,
        60% {
            transform: rotate(-25deg);
        }
        40%,
        80% {
            transform: rotate(10deg);
        }
    }
</style>

<a href="https://github.com/Aunean-ls" class="github-corner tooltipped hide-on-med-and-down" target="_blank"
   data-tooltip="Fork Me" data-position="left" data-delay="50">
    <svg viewBox="0 0 250 250" aria-hidden="true">
        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path>
        <path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
              fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path>
        <path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
              fill="currentColor" class="octo-body"></path>
    </svg>
</a>
        
    </nav>

</header>

    
<script src="/libs/cryptojs/crypto-js.min.js"></script>
<script>
    (function() {
        let pwd = '';
        if (pwd && pwd.length > 0) {
            if (pwd !== CryptoJS.SHA256(prompt('请输入访问本文章的密码')).toString(CryptoJS.enc.Hex)) {
                alert('密码错误，将返回主页！');
                location.href = '/';
            }
        }
    })();
</script>




<div class="bg-cover pd-header post-cover" style="background-image: url('https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u020.webp')">
    <div class="container" style="right: 0px;left: 0px;">
        <div class="row">
            <div class="col s12 m12 l12">
                <div class="brand">
                    <h1 class="description center-align post-title">Hive笔记</h1>
                </div>
            </div>
        </div>
    </div>
</div>




<main class="post-container content">

    
    <link rel="stylesheet" href="/libs/tocbot/tocbot.css">
<style>
    #articleContent h1::before,
    #articleContent h2::before,
    #articleContent h3::before,
    #articleContent h4::before,
    #articleContent h5::before,
    #articleContent h6::before {
        display: block;
        content: " ";
        height: 100px;
        margin-top: -100px;
        visibility: hidden;
    }

    #articleContent :focus {
        outline: none;
    }

    .toc-fixed {
        position: fixed;
        top: 64px;
    }

    .toc-widget {
        width: 345px;
        padding-left: 20px;
    }

    .toc-widget .toc-title {
        padding: 35px 0 15px 17px;
        font-size: 1.5rem;
        font-weight: bold;
        line-height: 1.5rem;
    }

    .toc-widget ol {
        padding: 0;
        list-style: none;
    }

    #toc-content {
        padding-bottom: 30px;
        overflow: auto;
    }

    #toc-content ol {
        padding-left: 10px;
    }

    #toc-content ol li {
        padding-left: 10px;
    }

    #toc-content .toc-link:hover {
        color: #42b983;
        font-weight: 700;
        text-decoration: underline;
    }

    #toc-content .toc-link::before {
        background-color: transparent;
        max-height: 25px;

        position: absolute;
        right: 23.5vw;
        display: block;
    }

    #toc-content .is-active-link {
        color: #42b983;
    }

    #floating-toc-btn {
        position: fixed;
        right: 15px;
        bottom: 76px;
        padding-top: 15px;
        margin-bottom: 0;
        z-index: 998;
    }

    #floating-toc-btn .btn-floating {
        width: 48px;
        height: 48px;
    }

    #floating-toc-btn .btn-floating i {
        line-height: 48px;
        font-size: 1.4rem;
    }
</style>
<div class="row">
    <div id="main-content" class="col s12 m12 l9">
        <!-- 文章内容详情 -->
<div id="artDetail">
    <div class="card">
        <div class="card-content article-info">
            <div class="row tag-cate">
                <div class="col s7">
                    
                    <div class="article-tag">
                        
                            <a href="/tags/Hive/">
                                <span class="chip bg-color">Hive</span>
                            </a>
                        
                    </div>
                    
                </div>
                <div class="col s5 right-align">
                    
                    <div class="post-cate">
                        <i class="fas fa-bookmark fa-fw icon-category"></i>
                        
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                学习笔记
                            </a>
                        
                    </div>
                    
                </div>
            </div>

            <div class="post-info">
                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-minus fa-fw"></i>发布日期:&nbsp;&nbsp;
                    2021-05-31
                </div>
                

                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-check fa-fw"></i>更新日期:&nbsp;&nbsp;
                    2021-08-04
                </div>
                

                
                <div class="info-break-policy">
                    <i class="far fa-file-word fa-fw"></i>文章字数:&nbsp;&nbsp;
                    6.4k
                </div>
                

                

                
            </div>
        </div>
        <hr class="clearfix">

        

        

        <div class="card-content article-card-content">
            <div id="articleContent">
                <h1 id="Hive笔记"><a href="#Hive笔记" class="headerlink" title="Hive笔记"></a>Hive笔记</h1><h2 id="第一章-Hive数据类型"><a href="#第一章-Hive数据类型" class="headerlink" title="第一章 Hive数据类型"></a>第一章 Hive数据类型</h2><h3 id="1-STRUCT类型的定义与指定分隔符"><a href="#1-STRUCT类型的定义与指定分隔符" class="headerlink" title="1.STRUCT类型的定义与指定分隔符"></a>1.STRUCT类型的定义与指定分隔符</h3><pre class=" language-sql"><code class="language-sql">info struct<span class="token operator">&lt;</span>name:string<span class="token punctuation">,</span>sex:string<span class="token punctuation">,</span>age:<span class="token keyword">int</span><span class="token operator">></span>
collection items <span class="token keyword">terminated by</span> <span class="token string">'集合数据项分隔符'</span><span class="token punctuation">;</span>
</code></pre>
<h3 id="2-MAP类型的定义与指定分隔符"><a href="#2-MAP类型的定义与指定分隔符" class="headerlink" title="2.MAP类型的定义与指定分隔符"></a>2.MAP类型的定义与指定分隔符</h3><pre class=" language-sql"><code class="language-sql">perf map<span class="token operator">&lt;</span>string<span class="token punctuation">,</span><span class="token keyword">int</span><span class="token operator">></span>
collection items <span class="token keyword">terminated by</span> <span class="token string">'集合数据项分隔符'</span> map <span class="token keyword">keys</span> <span class="token keyword">terminated by</span> <span class="token string">'键值对分隔符'</span>
</code></pre>
<h3 id="3-ARRAY类型的定义与指定分隔符"><a href="#3-ARRAY类型的定义与指定分隔符" class="headerlink" title="3.ARRAY类型的定义与指定分隔符"></a>3.ARRAY类型的定义与指定分隔符</h3><pre class=" language-sql"><code class="language-sql">emp_id_list array<span class="token operator">&lt;</span><span class="token keyword">int</span><span class="token operator">></span>
collection items terminate <span class="token keyword">by</span> <span class="token string">'集合数据项分隔符'</span>
</code></pre>
<table>
<thead>
<tr>
<th>数据类型</th>
<th>描述</th>
<th>示例</th>
</tr>
</thead>
<tbody><tr>
<td>STRUCT</td>
<td>一组命名字段，字段类型可以不同，可以通过’点‘符号访问元素内容。</td>
<td>Struct(‘john’,’doe’)</td>
</tr>
<tr>
<td>MAP</td>
<td>Map是一组键-值对元素集合。键的类型必须是原子的。</td>
<td>Map(‘a’,1,’b’,2)</td>
</tr>
<tr>
<td>ARRAY</td>
<td>数组是一组具有相同类型和名称的变量的集合。这些变量成为数组的元素，每组元素都有一个编号，编号从零开始。</td>
<td>Array(‘john’,’doe’)</td>
</tr>
</tbody></table>
<p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201019110909279.png" alt="image-20201019110909279"></p>
<p><img src="https://p.ananas.chaoxing.com/star3/origin/23d6178eea66b6979d912b09a5101ead.png" alt="img"></p>
<h2 id="第二章-数据定义与操作"><a href="#第二章-数据定义与操作" class="headerlink" title="第二章 数据定义与操作"></a>第二章 数据定义与操作</h2><h3 id="2-1-数据定义语言"><a href="#2-1-数据定义语言" class="headerlink" title="2.1 数据定义语言"></a>2.1 数据定义语言</h3><p>####2.1.1 数据库操作</p>
<p>#####1.创建数据库</p>
<ul>
<li>创建数据库：create database ‘数据库名’;</li>
<li>创建数据库并增加描述信息：create database ‘数据库名’ comment ‘描述信息’;</li>
<li>创建数据库并指定位置：create database ‘数据库名’ location ‘数据库位置’;</li>
<li>创建数据库并增加其他信息：create database ‘数据库名’ with dbproperties(‘键’=值);</li>
</ul>
<h5 id="2-修改数据库"><a href="#2-修改数据库" class="headerlink" title="2.修改数据库"></a>2.修改数据库</h5><ul>
<li>alter database 数据库名 set dbproperties(‘creator’=’xxx’,’data’=’xxxx-xx-xx’);</li>
</ul>
<h5 id="3-删除数据库"><a href="#3-删除数据库" class="headerlink" title="3.删除数据库"></a>3.删除数据库</h5><ul>
<li>删除空数据库<ul>
<li>drop database 数据库名;</li>
</ul>
</li>
<li>删除非空数据库<ul>
<li>drop database 数据库名 cascade;</li>
</ul>
</li>
</ul>
<p>####2.1.2 表操作</p>
<h5 id="1-创建表"><a href="#1-创建表" class="headerlink" title="1.创建表"></a>1.创建表</h5><pre><code>create table 数据库名.表名(
字段1 类型 comment '字段描述信息',
字段2 类型 comment '字段描述信息'
)
row format delimited fields terminated by '字符分隔符'
comment '表描述信息'
location '表位置';
</code></pre>
<ul>
<li>拷贝已经存在的表<ul>
<li>create table 表名 like 表名2;</li>
</ul>
</li>
</ul>
<h5 id="2-查看表"><a href="#2-查看表" class="headerlink" title="2.查看表"></a>2.查看表</h5><ul>
<li><p>查看所有表</p>
<ul>
<li>show tables;</li>
</ul>
</li>
<li><p>使用正则过滤出需要的表名</p>
<ul>
<li>show tables emp.*;</li>
</ul>
</li>
<li><p>查看表的详细结构信息</p>
<ul>
<li>desc 表名;</li>
</ul>
</li>
<li><p>查看表的位置 </p>
<ul>
<li>desc extended 表名</li>
</ul>
</li>
</ul>
<h5 id="3-创建外部表"><a href="#3-创建外部表" class="headerlink" title="3.创建外部表"></a>3.创建外部表</h5><pre class=" language-sql"><code class="language-sql"><span class="token keyword">create</span> external <span class="token keyword">table</span> 表名<span class="token punctuation">(</span>
字段 类型<span class="token punctuation">,</span>
字段 类型
<span class="token punctuation">)</span>
<span class="token keyword">row</span> format delimited <span class="token keyword">fields</span> <span class="token keyword">terminated by</span> <span class="token string">'字段分隔符'</span> <span class="token punctuation">[</span>location <span class="token string">'文件位置'</span><span class="token punctuation">]</span><span class="token punctuation">;</span>
</code></pre>
<h5 id="4-修改表"><a href="#4-修改表" class="headerlink" title="4.修改表"></a>4.修改表</h5><ul>
<li>表重命名<ul>
<li>alter table 表名 rename to 新表名;</li>
</ul>
</li>
<li>新增列信息<ul>
<li>alter table 表名 add columns(字段 类型 [,字段 类型]);</li>
</ul>
</li>
<li>修改类名、类型<ul>
<li>alter table 表名 change column 原列名 新列名 类型;</li>
</ul>
</li>
</ul>
<h5 id="5-删除表"><a href="#5-删除表" class="headerlink" title="5.删除表"></a>5.删除表</h5><ul>
<li>drop table 表名</li>
<li>清空表数据<ul>
<li>truncate table 表名; (外部表不行)</li>
</ul>
</li>
</ul>
<h3 id="2-2-分区表"><a href="#2-2-分区表" class="headerlink" title="2.2 分区表"></a>2.2 分区表</h3><h4 id="2-2-1-静态分区表"><a href="#2-2-1-静态分区表" class="headerlink" title="2.2.1 静态分区表"></a>2.2.1 静态分区表</h4><h5 id="1-创建静态分区表"><a href="#1-创建静态分区表" class="headerlink" title="1.创建静态分区表"></a>1.创建静态分区表</h5><pre class=" language-sql"><code class="language-sql"><span class="token keyword">create</span> <span class="token keyword">table</span> 表名<span class="token punctuation">(</span>
字段 类型<span class="token punctuation">[</span><span class="token punctuation">,</span>字段 类型<span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">]</span>
<span class="token punctuation">)</span>
partitioned <span class="token keyword">by</span><span class="token punctuation">(</span>分区字段 字段类型<span class="token punctuation">[</span><span class="token punctuation">,</span>分区字段 字段类型<span class="token punctuation">]</span><span class="token punctuation">)</span>
<span class="token keyword">row</span> format delimited <span class="token keyword">fields</span> <span class="token keyword">terminated by</span> <span class="token string">'字段分隔符'</span><span class="token punctuation">;</span>
</code></pre>
<ul>
<li>向分区表加载数据：<ul>
<li>load data local inpath ‘文件位置’ into table 表名 partition(分区字段=值[,分区字段=值]);</li>
</ul>
</li>
</ul>
<blockquote>
<ol>
<li>查看数据：select * from 表名;</li>
<li>查看分区：show partitions 表名;</li>
<li>查看hdfs分布式文件系统，看到可以反映区结构的子目录<ul>
<li>hdfs shell 命令查看</li>
<li>walker:50070 浏览器查看文件系统</li>
</ul>
</li>
</ol>
</blockquote>
<h4 id="2-2-2-动态分区表"><a href="#2-2-2-动态分区表" class="headerlink" title="2.2.2 动态分区表"></a>2.2.2 动态分区表</h4><blockquote>
<p>创建分区表，要先打开允许分区列是动态的。</p>
<p>set hive.exec.dynamic.partition=true;</p>
<p>set hive.exec.dynamic.partition=nonstrict;</p>
<p>set hive.exec.max.dynamic.partitions.pernode=1000;</p>
<p>set hive.mapred.mode=nonstrict;</p>
</blockquote>
<h5 id="1-写入数据到动态分区表"><a href="#1-写入数据到动态分区表" class="headerlink" title="1.写入数据到动态分区表"></a>1.写入数据到动态分区表</h5><pre class=" language-sql"><code class="language-sql"><span class="token keyword">insert</span> overwrite <span class="token keyword">table</span> 表名 <span class="token keyword">partition</span><span class="token punctuation">(</span>分区字段<span class="token punctuation">[</span><span class="token punctuation">,</span>分区字段<span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">]</span><span class="token punctuation">)</span> <span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<h5 id="2-修改分区"><a href="#2-修改分区" class="headerlink" title="2.修改分区"></a>2.修改分区</h5><ul>
<li>增加分区<ul>
<li>alter table 表名 add partition(分区字段=值[,分区字段=值]) location ‘数据位置’;</li>
</ul>
</li>
<li>修改分区的路径<ul>
<li>alter table 表名 partition(字段=值[,字段=值]) set location ‘数据位置’;</li>
</ul>
</li>
<li>删除某个分区的路径<ul>
<li>alter table 表名 drop partition(字段=值[,字段=值]);</li>
</ul>
</li>
</ul>
<p>###2.3 数据操作语言</p>
<h4 id="2-3-1-insert语句"><a href="#2-3-1-insert语句" class="headerlink" title="2.3.1 insert语句"></a>2.3.1 insert语句</h4><ul>
<li>向表中插入数据<ul>
<li>insert overwrite table 表名 select */字段 from 表名2[where 语句];</li>
</ul>
</li>
<li>向分区表插入数据<ul>
<li>insert overwrite table 表名 partition(分区字段=值[,分区字段2=值2]) select * from 表名2[where语句];</li>
</ul>
</li>
<li>动态分区数据插入<ul>
<li>insert overwrite table 表名 partition(分区字段1[,分区字段2…]) select 字段1[,字段2…] from 表名;</li>
</ul>
</li>
<li>混合使用动态分区和静态分区数据插入<ul>
<li>insert overwrite table 表名 partition(分区字段=值[,分区字段2…]) select 字段1[字段2…] from 表名;</li>
</ul>
</li>
</ul>
<h4 id="2-3-2-创建备份表并向其加载数据"><a href="#2-3-2-创建备份表并向其加载数据" class="headerlink" title="2.3.2 创建备份表并向其加载数据"></a>2.3.2 创建备份表并向其加载数据</h4><pre class=" language-sql"><code class="language-sql"><span class="token keyword">create</span> <span class="token keyword">table</span> 表名 <span class="token keyword">as</span> <span class="token keyword">select</span> <span class="token operator">*</span><span class="token operator">/</span>字段 <span class="token keyword">from</span> 表名<span class="token number">2</span><span class="token punctuation">;</span>
</code></pre>
<h4 id="2-3-3-数据的导入与导出"><a href="#2-3-3-数据的导入与导出" class="headerlink" title="2.3.3 数据的导入与导出"></a>2.3.3 数据的导入与导出</h4><ul>
<li>导入<ul>
<li>dfs -put 本地数据文件 hdfs 目录;</li>
</ul>
</li>
<li>导出<ul>
<li>insert overwrite local directory ‘本地文件位置’ select */字段 from 表名 where 语句;</li>
<li>dfs -get hdfs数据文件 本地目录;</li>
</ul>
</li>
</ul>
<h3 id="2-4-分桶表操作"><a href="#2-4-分桶表操作" class="headerlink" title="2.4 分桶表操作"></a>2.4 分桶表操作</h3><ul>
<li><p>分桶，就是将数据按照指定的字段进行划分到多个文件当中去，==分桶就是MapReduce中的分区==。必须配合内部表或外部表一起使用。</p>
</li>
<li><p>好处：避免单个表文件数据过大的问题。</p>
</li>
<li><p>开启 Hive 的分桶功能：</p>
</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>enforce<span class="token punctuation">.</span>bucketing<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ul>
<li>设置 Reduce 个数：</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>job<span class="token punctuation">.</span>reduces<span class="token operator">=</span><span class="token number">3</span><span class="token punctuation">;</span>
</code></pre>
<ul>
<li>创建分桶表</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">create</span> <span class="token keyword">table</span> course<span class="token punctuation">(</span>c_id string<span class="token punctuation">,</span>c_name string<span class="token punctuation">,</span>t_id string<span class="token punctuation">)</span> <span class="token keyword">clustered</span> <span class="token keyword">by</span><span class="token punctuation">(</span>c_id<span class="token punctuation">)</span> <span class="token keyword">into</span> <span class="token number">3</span> buckets <span class="token keyword">row</span> format delimited <span class="token keyword">terminated by</span> <span class="token string">'\t'</span><span class="token punctuation">;</span>
</code></pre>
<blockquote>
<p>分桶表的数据加载，由于通标的数据加载通过 hdfs dfs -put 文件 或者通过 load data 均不好使，只能通过 insert overwrite</p>
<p>创建普通表，并通过 insert overwrite 的方式将普通表的数据同构查询的方式加载到桶表当中去。</p>
</blockquote>
<ul>
<li>创建普通表</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">create</span> <span class="token keyword">table</span> course_common<span class="token punctuation">(</span>c_string<span class="token punctuation">,</span>c_name string<span class="token punctuation">,</span>t_id string<span class="token punctuation">)</span> <span class="token keyword">row</span> format delimited <span class="token keyword">fields</span> <span class="token keyword">terminated by</span> <span class="token string">'\t'</span><span class="token punctuation">;</span>
</code></pre>
<ul>
<li>普通表中加载数据</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">load</span> <span class="token keyword">data</span> <span class="token keyword">local</span> inpath <span class="token string">'/root/hivedata/course_txt'</span> <span class="token keyword">into</span> <span class="token keyword">table</span> course_common<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>通过 insert overwrite 给桶表中加载数据</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">insert</span> overwrite <span class="token keyword">table</span> course <span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> course_common cluster <span class="token keyword">by</span><span class="token punctuation">(</span>c_id<span class="token punctuation">)</span><span class="token punctuation">;</span>
</code></pre>
<h3 id="2-5-Hive查询语法"><a href="#2-5-Hive查询语法" class="headerlink" title="2.5 Hive查询语法"></a>2.5 Hive查询语法</h3><h4 id="2-5-1-SELECT"><a href="#2-5-1-SELECT" class="headerlink" title="2.5.1 SELECT"></a>2.5.1 SELECT</h4><pre class=" language-SQL"><code class="language-SQL">select [all | distinct] select_expr,select_expr,...
from table_reference
[where where_condition]
[group by col_list [having condition]]
[cluster by col_list | 
[distribute by col_list] | 
[order by col_list]
]
[limit number]
</code></pre>
<ol>
<li>order by 会对输入做全局排序，因此只有一个 reducer，会导致输入规模较大时，需要较长的计算时间。</li>
<li>sort by 不是全局排序，其在数据进入 reducer 前完成排序。因此，如果用 sort by 进行排序，并且设置 mapred.reduce.tasks&gt;1,则 sort by 只保证每个 reducer 的输出有序，不保证全局有序。</li>
<li>distribute by 根据指定的字段将数据分到不同的 reducer，且分发算法是 hash 散列。</li>
<li>cluster by 除了具有 distribute by 的功能外，还会对该字段进行排序。</li>
</ol>
<p>因此，如果 distribute 和 sort 字段是同一个时，此时，==cluster by = distribute by + sort by==</p>
<p>####2.5.2 常用函数</p>
<ul>
<li>求总行数（count）</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">count</span><span class="token punctuation">(</span><span class="token operator">*</span><span class="token punctuation">)</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>求分数的最大值（max）</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">max</span><span class="token punctuation">(</span>字段<span class="token punctuation">)</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>求分数的最小值（min）</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">min</span><span class="token punctuation">(</span>字段<span class="token punctuation">)</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>求分数的总和（sum）</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">sum</span><span class="token punctuation">(</span>字段<span class="token punctuation">)</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>求分数的平均值（avg）</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> ave<span class="token punctuation">(</span>字段<span class="token punctuation">)</span> <span class="token keyword">from</span> 表名<span class="token punctuation">;</span>
</code></pre>
<h4 id="2-5-3-LIMIT"><a href="#2-5-3-LIMIT" class="headerlink" title="2.5.3 LIMIT"></a>2.5.3 LIMIT</h4><p>典型的查询会返回多行数据。limit子句用于限制返回的行数。</p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> 表名 <span class="token keyword">limit</span> <span class="token number">3</span><span class="token punctuation">;</span>
</code></pre>
<h4 id="2-5-4-WHERE"><a href="#2-5-4-WHERE" class="headerlink" title="2.5.4 WHERE"></a>2.5.4 WHERE</h4><pre class=" language-SQL"><code class="language-SQL">select * from 表名 where ...;
</code></pre>
<h4 id="2-5-5-LIKE-和-RLIKE"><a href="#2-5-5-LIKE-和-RLIKE" class="headerlink" title="2.5.5 LIKE 和 RLIKE"></a>2.5.5 LIKE 和 RLIKE</h4><ol>
<li>使用like运算选择类似的值。</li>
<li>选择条件可以包含字符或数字。</li>
<li>RLIKE 子句是 Hive 中这个功能的一个扩展，其可以通过Java的正则表达式这个更强大的语言来指定匹配条件。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token operator">%</span> 代表零个或多个字符（任意字符）
_ 代表一个字符
</code></pre>
<h4 id="2-5-6-逻辑运算符"><a href="#2-5-6-逻辑运算符" class="headerlink" title="2.5.6 逻辑运算符"></a>2.5.6 逻辑运算符</h4><table>
<thead>
<tr>
<th>操作符</th>
<th>含义</th>
</tr>
</thead>
<tbody><tr>
<td>AND</td>
<td>逻辑并</td>
</tr>
<tr>
<td>OR</td>
<td>逻辑或</td>
</tr>
<tr>
<td>NOT</td>
<td>逻辑否</td>
</tr>
</tbody></table>
<h4 id="2-5-7-分组"><a href="#2-5-7-分组" class="headerlink" title="2.5.7 分组"></a>2.5.7 分组</h4><p><strong>GROUP BY 语句</strong></p>
<blockquote>
<p>查看字段必须是分组字段</p>
</blockquote>
<ul>
<li>计算每个学生的平均分数</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s_id<span class="token punctuation">,</span><span class="token function">avg</span><span class="token punctuation">(</span>s_score<span class="token punctuation">)</span> <span class="token keyword">from</span> tableName <span class="token keyword">group</span> <span class="token keyword">by</span> s_id<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>计算每个学生最高成绩</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s_id<span class="token punctuation">,</span><span class="token function">max</span><span class="token punctuation">(</span>s_score<span class="token punctuation">)</span> <span class="token keyword">from</span> tableName <span class="token keyword">group</span> <span class="token keyword">by</span> s_id<span class="token punctuation">;</span>
</code></pre>
<h4 id="2-5-8-HIVING语句"><a href="#2-5-8-HIVING语句" class="headerlink" title="2.5.8 HIVING语句"></a>2.5.8 HIVING语句</h4><ol>
<li><p>hiving与where不同点</p>
<ol>
<li>where针对表中的列发挥作用，查询数据；hiving针对查询结果中的列发挥作用，筛选数据。</li>
<li>where后面不能写分组函数，而having后面可以使用分组函数。</li>
<li>hiving只用于group by分组统计语句。</li>
</ol>
</li>
<li><p>案例实操：</p>
<ul>
<li>求平均分大于八十五的学生</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s_id<span class="token punctuation">,</span><span class="token function">avg</span><span class="token punctuation">(</span>s_score<span class="token punctuation">)</span> avg_score <span class="token keyword">from</span> tableName <span class="token keyword">group</span> <span class="token keyword">by</span> s_id <span class="token keyword">having</span> avg_score <span class="token operator">></span> <span class="token number">85</span><span class="token punctuation">;</span>
</code></pre>
</li>
</ol>
<h4 id="2-5-9-JOIN-语句"><a href="#2-5-9-JOIN-语句" class="headerlink" title="2.5.9 JOIN 语句"></a>2.5.9 JOIN 语句</h4><p>hive支持通常的SQL   JOIN语句，但是只支持等值连接，不支持非等值连接<br>案例操作：</p>
<ul>
<li>查询分数对应的姓名</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s<span class="token punctuation">.</span>s_id<span class="token punctuation">,</span>s<span class="token punctuation">.</span>s_score<span class="token punctuation">,</span>stu<span class="token punctuation">.</span>s_name<span class="token punctuation">,</span>stu<span class="token punctuation">.</span>s_birth <span class="token keyword">from</span> score s <span class="token keyword">join</span> student stu <span class="token keyword">on</span> s<span class="token punctuation">.</span>s_id <span class="token operator">=</span> stu<span class="token punctuation">.</span>s_id<span class="token punctuation">;</span>
</code></pre>
<ul>
<li>合并老师与课程表</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> teacher t <span class="token keyword">join</span> course <span class="token number">c</span> <span class="token keyword">on</span> t<span class="token punctuation">.</span>t_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>t_id<span class="token punctuation">;</span>
</code></pre>
<pre class=" language-sql"><code class="language-sql">内连接：只有进行连接的两个表中都存在与连接条件相匹配的数据才会被保留下来。
<span class="token keyword">select</span> <span class="token operator">*</span> form teacher t <span class="token keyword">inner</span> <span class="token keyword">join</span> course <span class="token number">c</span> <span class="token keyword">on</span> t<span class="token punctuation">.</span>t_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>t_id<span class="token punctuation">;</span>

左外连接：<span class="token keyword">join</span>操作符左边表中符合<span class="token keyword">WHERE</span>子句的所有记录将会被返回。
<span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> teacher t <span class="token keyword">left</span> <span class="token keyword">join</span> course <span class="token number">c</span> <span class="token keyword">on</span> t<span class="token punctuation">.</span>t_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>t_id<span class="token punctuation">;</span>

右外连接：<span class="token keyword">join</span>操作符右边表符合<span class="token keyword">WHERE</span>子句的所有记录将会被返回。
<span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> teacher t <span class="token keyword">right</span> <span class="token keyword">join</span> corese <span class="token number">c</span> <span class="token keyword">on</span> t<span class="token punctuation">.</span>t_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>t_id<span class="token punctuation">;</span>

多表连接：连接n个表，至少需要n<span class="token number">-1</span>个连接条件。
<span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> teacher t
<span class="token keyword">left</span> <span class="token keyword">join</span> course <span class="token number">c</span>
<span class="token keyword">on</span> t<span class="token punctuation">.</span>t_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>t_id
<span class="token keyword">left</span> <span class="token keyword">join</span> score s
<span class="token keyword">on</span> s<span class="token punctuation">.</span>c_id <span class="token operator">=</span> <span class="token number">c</span><span class="token punctuation">.</span>c_id
<span class="token keyword">left</span> <span class="token keyword">join</span> student stu
<span class="token keyword">on</span> s<span class="token punctuation">.</span>s_id <span class="token operator">=</span> stu<span class="token punctuation">.</span>s_id<span class="token punctuation">;</span>
</code></pre>
<h4 id="2-5-10-排序"><a href="#2-5-10-排序" class="headerlink" title="2.5.10 排序"></a>2.5.10 排序</h4><p><strong>全局排序</strong></p>
<p>order by：全局排序，一个reduce</p>
<ol>
<li><p>使用 ORDER B Y子句排序 ASC:升序，DESC:降序</p>
</li>
<li><p>ORDER BY 子句在 SELECT 语句的结尾。</p>
</li>
<li><p>案例实操</p>
<ol>
<li>查询学生的成绩，并按照分数降序（升序）排列</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> student s <span class="token keyword">left</span> <span class="token keyword">join</span> score sco <span class="token keyword">on</span> s<span class="token punctuation">.</span>s_id <span class="token operator">=</span> sco<span class="token punctuation">.</span>s_id <span class="token keyword">order</span> <span class="token keyword">by</span> sco<span class="token punctuation">.</span>s_score <span class="token keyword">desc</span><span class="token punctuation">(</span><span class="token keyword">asc</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
</code></pre>
</li>
</ol>
<p><strong>按照别名排序</strong></p>
<ul>
<li>按照分数的平均值排序</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s_id<span class="token punctuation">,</span><span class="token function">avg</span><span class="token punctuation">(</span>s_score<span class="token punctuation">)</span> avg <span class="token keyword">from</span> score <span class="token keyword">group</span> <span class="token keyword">by</span> s_id <span class="token keyword">order</span> <span class="token keyword">by</span> avg<span class="token punctuation">;</span>
</code></pre>
<p><strong>多个列排序</strong></p>
<ul>
<li>按照学生id和平均成绩进行排序</li>
</ul>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> s_id<span class="token punctuation">,</span><span class="token function">avg</span><span class="token punctuation">(</span>s_score<span class="token punctuation">)</span> avg <span class="token keyword">from</span> score <span class="token keyword">group</span> <span class="token keyword">by</span> s_id <span class="token keyword">order</span> <span class="token keyword">by</span> s_id<span class="token punctuation">,</span>avg<span class="token punctuation">;</span>
</code></pre>
<p><strong>每个MapReduce内部排序（sort by）局部排序</strong></p>
<p>sort by：每个MapReduce内部进行排序，对全局结果集来说不是排序。</p>
<ol>
<li>设置reduce个数</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>job<span class="token punctuation">.</span>reduces<span class="token operator">=</span><span class="token number">3</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>查看设置reduce个数</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>job<span class="token punctuation">.</span>reduces<span class="token punctuation">;</span>
</code></pre>
<p><strong>分区排序（DISTRIBUTE BY）</strong></p>
<p>distribute by：类似MR中partititon，进行分区，结合sort by使用。</p>
<p>注意，Hive要求distribute by语句要写在sort by语句之前。</p>
<p>对于distribute by进行测试，一定要分配多reduce进行处理，否则无法看到distribute by的效果。</p>
<p>案例实操：先按照学生id进行分区，再按照学生成绩进行排序。</p>
<ol>
<li>设置reduce的个数，将我们对应的s_id划分到对应的reduce当中去</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>job<span class="token punctuation">.</span>reduces<span class="token operator">=</span><span class="token number">7</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>通过distribute by进行数据的分区</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">insert</span> overwrite <span class="token keyword">local</span> directory <span class="token string">'export/servers/hivedatas/sort'</span> <span class="token keyword">select</span> <span class="token operator">*</span><span class="token keyword">from</span> score distribute <span class="token keyword">by</span> spid sort <span class="token keyword">by</span> s_score<span class="token punctuation">;</span>
</code></pre>
<p>##第三章 Hive函数</p>
<h3 id="3-1-内置函数"><a href="#3-1-内置函数" class="headerlink" title="3.1 内置函数"></a>3.1 内置函数</h3><p><a target="_blank" rel="noopener" href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF">Hive官方文档</a></p>
<ol>
<li>查看系统自带的函数</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">show</span> functions<span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>显示自带的函数的用法</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">desc</span> <span class="token keyword">function</span> upper<span class="token punctuation">;</span>
</code></pre>
<ol start="3">
<li>详细显示自带的函数的用法</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">desc</span> <span class="token keyword">function</span> <span class="token keyword">extended</span> upper<span class="token punctuation">;</span>
</code></pre>
<ol start="4">
<li>常用的内置函数</li>
</ol>
<pre class=" language-ruby"><code class="language-ruby"><span class="token comment" spellcheck="true">#字符串连接函数:concat</span>
select <span class="token function">concat</span><span class="token punctuation">(</span><span class="token string">'abc'</span><span class="token punctuation">,</span> <span class="token string">'def'</span><span class="token punctuation">,</span> <span class="token string">'gh'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment" spellcheck="true">#带分隔符字符串连接函数:concat_ws</span>
select <span class="token function">concat_ws</span><span class="token punctuation">(</span><span class="token string">'.'</span><span class="token punctuation">,</span> <span class="token string">'abc'</span><span class="token punctuation">,</span> <span class="token string">'def'</span><span class="token punctuation">,</span> <span class="token string">'gh'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment" spellcheck="true">#cast类型转换</span>
select <span class="token function">cast</span><span class="token punctuation">(</span><span class="token number">1.5</span> as int<span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment" spellcheck="true">#get_json_object(json解析函数，用来处理json，必须是json格式)</span>
select get<span class="token punctuation">.</span><span class="token function">json_object</span><span class="token punctuation">(</span><span class="token string">'{"name": "jack", "age": "20"}'</span><span class="token punctuation">,</span> <span class="token string">'$.name'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment" spellcheck="true">#URL解析函数</span>
select <span class="token function">parse_url</span><span class="token punctuation">(</span><span class="token string">'http://facebook.com/path1/p.php?k1=v1&amp;k2=v2#Ref1'</span><span class="token punctuation">,</span><span class="token string">'HOST'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
select <span class="token function">parse_url</span><span class="token punctuation">(</span><span class="token string">'http://facebook.com/path1/p.php?k1=v1&amp;k2=v2#Ref1'</span><span class="token punctuation">,</span><span class="token string">'PATH'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
select <span class="token function">parse_url</span><span class="token punctuation">(</span><span class="token string">'http://facebook.com/path1/p.php?k1=v1&amp;k2=v2#Ref1'</span><span class="token punctuation">,</span><span class="token string">'QUERY'</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token comment" spellcheck="true">#explode:把map集合中每个键值对或数组中的每个元素都单独生成一行的形式</span>
</code></pre>
<h3 id="3-2-自定义函数"><a href="#3-2-自定义函数" class="headerlink" title="3.2 自定义函数"></a>3.2 自定义函数</h3><h4 id="3-2-1-概述"><a href="#3-2-1-概述" class="headerlink" title="3.2.1 概述"></a>3.2.1 概述</h4><ol>
<li><p>Hive自带了一些函数，比如：max/min等，当Hive提供的内置函数无法满足你的业务处理需要时，此时就可以考虑使用用户自定义函数(UDF)。</p>
</li>
<li><p>根据用户自定义函数类别分为以下三种：</p>
<ol>
<li>UDF（User-Defined-Function）<ul>
<li>一进一出</li>
</ul>
</li>
<li>UDAF（User-Defined Aggregation Function）<ul>
<li>聚集函数，多进一出</li>
<li>类似于：==count / max / min==</li>
</ul>
</li>
<li>UDTF（User-Defined Table-Generating Functions）<ul>
<li>一进多出</li>
<li>如：==lateral / view / explore()==</li>
</ul>
</li>
</ol>
</li>
<li><p>编程步骤：</p>
<ol>
<li>继承org.apache.hadoop.hive.ql.UDF</li>
<li>需要实现evaluate函数；evaluate函数支持重载</li>
</ol>
</li>
<li><p>注意事项</p>
<ol>
<li>UDF必须要有返回类型，可以返回null，但是返回类型不能为void。</li>
<li>UDF中常用 Text / LongWritable 等类型，不推荐使用 java 类型。</li>
</ol>
</li>
</ol>
<h4 id="3-2-2-UDF-开发实例"><a href="#3-2-2-UDF-开发实例" class="headerlink" title="3.2.2 UDF 开发实例"></a>3.2.2 UDF 开发实例</h4><p><strong>step1 创建Maven工程</strong></p>
<p>##第四章 Hive的数据压缩</p>
<blockquote>
<p>在实际工作当中，hive当中处理的数据，一般都需要经过压缩，前期我们在学习hadoop的时候，已经配置过hadoop的压缩，我们这里的hive也是一样的可以使用压缩来节省我们的MR处理的网络带宽.</p>
</blockquote>
<h3 id="4-1-MR支持的压缩编码"><a href="#4-1-MR支持的压缩编码" class="headerlink" title="4.1 MR支持的压缩编码"></a>4.1 MR支持的压缩编码</h3><p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201029113639790.png" alt="image-20201029113639790"></p>
<p>为了支持多种压缩/解压缩算法，Hadoop引入了编码/解码器，如下表所示：</p>
<table>
<thead>
<tr>
<th>压缩格式</th>
<th>对应的编码/解码器</th>
</tr>
</thead>
<tbody><tr>
<td>DEFLATE</td>
<td>org.apache.hadoop.io.compress.DefaultCodec</td>
</tr>
<tr>
<td>gzip</td>
<td>org.apache.hadoop.io.compress.GzipCodec</td>
</tr>
<tr>
<td>bzip2</td>
<td>org.apache.hadoop.io.compress.BZip2Codec</td>
</tr>
<tr>
<td>LZO</td>
<td>com.hadoop.compression.lzo.LzopCodec</td>
</tr>
<tr>
<td>LZ4</td>
<td>org.apache.hadoop.io.compress.Lz4Codec</td>
</tr>
<tr>
<td>Snappy</td>
<td>org.apache.hadoop.io.compress.SnappyCoded</td>
</tr>
</tbody></table>
<h3 id="4-2-压缩配置参数"><a href="#4-2-压缩配置参数" class="headerlink" title="4.2 压缩配置参数"></a>4.2 压缩配置参数</h3><p>要在Hadoop中启用压缩，可以配置如下参数（mapred-site.xml文件中)：</p>
<table>
<thead>
<tr>
<th>参数</th>
<th>默认值</th>
<th>阶段</th>
<th>建议</th>
</tr>
</thead>
<tbody><tr>
<td>io.compression.codesc（在core-site.xml中配置）</td>
<td>1.org.apache.hadoop.io.compress.DefaultCoder    2.org.apache.hadoop.io.compress.GzipCodec            3.org.apache.hadoop.io.compress.BZip2Codec          4.org.apache.hadoop.io.compress.Lz4Codec</td>
<td>输入压缩</td>
<td>Hadoop使用文件扩展名判断是否支持某种编解码器</td>
</tr>
<tr>
<td>mapreduce.map.output.compress</td>
<td>false</td>
<td>mapper输出</td>
<td>这个参数设为true启用压缩</td>
</tr>
<tr>
<td>mapreduce.map.output.compress.codec</td>
<td>org.apache.hadoop.io.compress.DefaultCodec</td>
<td>mapper输出</td>
<td>snappy编解码器在此阶段压缩数据</td>
</tr>
<tr>
<td>mapreduce.output.fileoutputformat.compress</td>
<td>false</td>
<td>reducer输出</td>
<td>这个参数设为true启用压缩</td>
</tr>
<tr>
<td>mapreduce.output.fileoutputformat.compress.codec</td>
<td>org.apache.hadoop.io.compress.DefaultCodes</td>
<td>reducer输出</td>
<td>使用标准工具或者编解码器，如gzip和bzip2</td>
</tr>
<tr>
<td>mapreduce.output.fileoutputformat.compress.type</td>
<td>RECORD</td>
<td>reducer输出</td>
<td>SequenceFile输出使用的压缩类型：NONE和BLOCK</td>
</tr>
</tbody></table>
<h3 id="4-3-开启Map输出阶段压缩"><a href="#4-3-开启Map输出阶段压缩" class="headerlink" title="4.3 开启Map输出阶段压缩"></a>4.3 开启Map输出阶段压缩</h3><p>开启map输出阶段压缩可以减少job中map和Reduce task间数据传输量。具体配置如下：</p>
<p><strong>案例实操</strong></p>
<ol>
<li>开启hive中间传输数据压缩功能</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>compress<span class="token punctuation">.</span>intermediate<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>开启mapreduce中map输出压缩功能</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>map<span class="token punctuation">.</span>output<span class="token punctuation">.</span>compress<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="3">
<li>设置mapreduce中map输出数据的压缩方式</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>map<span class="token punctuation">.</span>output<span class="token punctuation">.</span>compress<span class="token punctuation">.</span>codec<span class="token operator">=</span>org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>hadoop<span class="token punctuation">.</span>io<span class="token punctuation">.</span>compress<span class="token punctuation">.</span>SnappyCodec<span class="token punctuation">;</span>
</code></pre>
<ol start="4">
<li>执行查询语句</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">count</span><span class="token punctuation">(</span><span class="token operator">*</span><span class="token punctuation">)</span> <span class="token keyword">from</span> tableName<span class="token punctuation">;</span>
</code></pre>
<h3 id="4-4-开启Reduce输出阶段压缩"><a href="#4-4-开启Reduce输出阶段压缩" class="headerlink" title="4.4 开启Reduce输出阶段压缩"></a>4.4 开启Reduce输出阶段压缩</h3><p>当Hive将输出写入到表中时，输出内容同样可以进行压缩。属性hive.exec.compress.output控制着这个功能。用户可能需要保持默认设置文件中的默认值false，这样默认的输出就是非压缩的纯文本文件了。用户可以通过在查询语句或执行脚本中设置这个值为true，来开启输出结果压缩功能。</p>
<p><strong>案例实操</strong></p>
<ol>
<li>开启hive最终输出数据压缩功能</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>compress<span class="token punctuation">.</span>output<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>开启mapreduce最终输出数据压缩</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>output<span class="token punctuation">.</span>fileoutputformat<span class="token punctuation">.</span>compress<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="3">
<li>设置mapreduce最终数据输出压缩方式</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>output<span class="token punctuation">.</span>fileoutputformat<span class="token punctuation">.</span>compress<span class="token punctuation">.</span>codec<span class="token operator">=</span>org<span class="token punctuation">.</span>apache<span class="token punctuation">.</span>hadoop<span class="token punctuation">.</span>io<span class="token punctuation">.</span>compress<span class="token punctuation">.</span>SnappyCodec<span class="token punctuation">;</span>
</code></pre>
<ol start="4">
<li>设置mapreduce最终数据输出压缩为块压缩</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapreduce<span class="token punctuation">.</span>output<span class="token punctuation">.</span>fileoutputformat<span class="token punctuation">.</span>compress<span class="token punctuation">.</span><span class="token keyword">type</span><span class="token operator">=</span>BLOCK<span class="token punctuation">;</span>
</code></pre>
<ol start="5">
<li>测试一下输出结果是否是压缩文件</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">insert</span> overwrite <span class="token keyword">local</span> directory <span class="token string">'/root/input_hivedata'</span> <span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> moviesp<span class="token punctuation">;</span>
</code></pre>
<h2 id="第五章-Hive调优"><a href="#第五章-Hive调优" class="headerlink" title="第五章 Hive调优"></a>第五章 Hive调优</h2><h3 id="5-1-Fetch抓取"><a href="#5-1-Fetch抓取" class="headerlink" title="5.1 Fetch抓取"></a>5.1 Fetch抓取</h3><p>Hive中对某些情况的查询可以不必使用MapReduce计算。例如:SELECT * FROM score;在这种情况下，Hive可以简单地读取score对应的存储目录下的文件，然后输出查询结果到控制台。<br><strong>通过设置hive.fetch.task.conversion参数,可以控制查询语句是否走MapReduce.</strong></p>
<p><strong>案例实操：</strong></p>
<ol>
<li>把 hive.fetch.task.conversion 设置成 none，然后执行查询语句，都会执行 mapreduce 程序。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">fetch</span><span class="token punctuation">.</span>task<span class="token punctuation">.</span>conversion<span class="token operator">=</span>none<span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>把 hive.fetch.task.conversion 设置成 more，然后执行查询语句，不会执行 mapreduce 程序。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">fetch</span><span class="token punctuation">.</span>task<span class="token punctuation">.</span>conversion<span class="token operator">=</span>more<span class="token punctuation">;</span>
</code></pre>
<h3 id="5-2-本地模式"><a href="#5-2-本地模式" class="headerlink" title="5.2 本地模式"></a>5.2 本地模式</h3><p>大多数的Hadoop Job是需要Hadoop提供的完整的可扩展性来处理大数据集的。不过，有时Hive的输入数据量是非常小的。在这种情况下，为查询触发执行任务时消耗可能会比实际job的执行时间要多的多。对于大多数这种情况，Hive可以通过本地模式在单台机器上处理所有的任务。对于小数据集，执行时间可以明显被缩短。</p>
<p>用户可以通过设置==hive.exec.mode.local.auto==的值为true，来让Hive在适当的时候自动启动这个优化。</p>
<p><strong>案例实操</strong></p>
<ol>
<li>开启本地模式，并执行查询语句</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>mode<span class="token punctuation">.</span><span class="token keyword">local</span><span class="token punctuation">.</span>auto<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
<span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> movies cluster <span class="token keyword">by</span> score<span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>关闭本地模式，并执行查询语句</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>mode<span class="token punctuation">.</span><span class="token keyword">local</span><span class="token punctuation">.</span>auto<span class="token operator">=</span><span class="token boolean">false</span><span class="token punctuation">;</span>
<span class="token keyword">select</span> <span class="token operator">*</span> <span class="token keyword">from</span> movies cluster <span class="token keyword">by</span> score<span class="token punctuation">;</span>
</code></pre>
<h3 id="5-3-MapJoin"><a href="#5-3-MapJoin" class="headerlink" title="5.3 MapJoin"></a>5.3 MapJoin</h3><p>如果不指定MapJoin或者不符合MapJoin的条件，那么Hive解析器会在Reduce阶段完成join,容易发生数据倾斜。可以用MapJoin把小表全部加载到内存在map端进行join，避免reducer处理。</p>
<ol>
<li><p>开启MapJoin参数设置：</p>
<ol>
<li>设置自动选择 MapJoin</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>auto<span class="token punctuation">.</span><span class="token keyword">convert</span><span class="token punctuation">.</span><span class="token keyword">join</span><span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>大表小表的阈值设置（默认25M以下认为是小表）：</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>mapjoin<span class="token punctuation">.</span>smalltable<span class="token punctuation">.</span>filesize<span class="token operator">=</span><span class="token number">25123456</span><span class="token punctuation">;</span>
</code></pre>
</li>
</ol>
<h3 id="5-4-Group-By"><a href="#5-4-Group-By" class="headerlink" title="5.4 Group By"></a>5.4 Group By</h3><p>默认情况下，Map阶段同一Key数据分发给一个reduce，当一个key数据过大时就倾斜了。并不是所有的聚合操作都需要在Reduce端完成，很多聚合操作都可以先在Map端进行部分聚合，最后在Reduce端得出最终结果。</p>
<p><strong>开启Map端聚合参数设置</strong></p>
<ol>
<li>是否在Map端进行聚合，默认为True</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>map<span class="token punctuation">.</span>aggr<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>在Map端进行聚合操作的条目数目</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>groupby<span class="token punctuation">.</span>mapaggr<span class="token punctuation">.</span>checkinterval<span class="token operator">=</span><span class="token number">100000</span><span class="token punctuation">;</span>
</code></pre>
<ol start="3">
<li>有数据倾斜的时候进行负载均衡（默认是false）</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>groupby<span class="token punctuation">.</span>skewindata<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<p>当选项设定为true，生成的查询计划会有两个MR Job。<br>第一个MR Job中，Map的输出结果会随机分布到Reduce中，每个Reduce做部分聚合操作，并输出结果，这样处理的结果是相同的Group By Key有可能被分发到不同的Reduce中，从而达到负载均衡的目的;<br>第二个MR .Job再根据预处理的数据结果按照Group By Key分布到Reduce中（这个过程可以保证相同的Group By Key被分布到同一个Reduce中)，最后完成最终的聚合操作。</p>
<h3 id="5-5-Count-distinct"><a href="#5-5-Count-distinct" class="headerlink" title="5.5 Count(distinct)"></a>5.5 Count(distinct)</h3><p>数据量小的时候无所谓，数据量大的情况下，由于COUNT DISTINCT操作需要用一个Reduce Task来完成，这一个Reduce需要处理的数据量太大，就会导致整个Job很难完成，一般COUNT DISTINCT使用先GROUP BY再COUNT的方式替换:</p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">select</span> <span class="token function">count</span><span class="token punctuation">(</span><span class="token keyword">distinct</span> keyword<span class="token punctuation">)</span> <span class="token keyword">from</span> sogou<span class="token punctuation">;</span>
<span class="token keyword">select</span> <span class="token function">count</span><span class="token punctuation">(</span>keyword<span class="token punctuation">)</span> <span class="token keyword">from</span> <span class="token punctuation">(</span><span class="token keyword">select</span> url <span class="token keyword">from</span> sogou <span class="token keyword">group</span> <span class="token keyword">by</span> keyword<span class="token punctuation">)</span> <span class="token number">a</span><span class="token punctuation">;</span>
</code></pre>
<h3 id="5-6-笛卡尔积"><a href="#5-6-笛卡尔积" class="headerlink" title="5.6 笛卡尔积"></a>5.6 笛卡尔积</h3><p>尽量避免笛卡尔积，即避免join的时候不加on条件，或者无效的on条件，Hive只能使用1个reducer来完成笛卡尔积。</p>
<h3 id="5-7-动态分区调整"><a href="#5-7-动态分区调整" class="headerlink" title="5.7 动态分区调整"></a>5.7 动态分区调整</h3><p>往hive分区表中插入数据时，hive提供了一个动态分区功能，其可以基于查询参数的位置去推断分区的名称，从而建立分区。使用Hive的动态分区，需要进行相应的配置。</p>
<p>Hive的动态分区是以第一个表的分区规则，来对应第二个表的分区规则，将第一个表的所有分区，全部拷贝到第二个表中来，第二个表在加载数据的时候，不需要指定分区了，直接用第一个表的分区即可</p>
<h4 id="5-7-1-开启动态分区参数设置"><a href="#5-7-1-开启动态分区参数设置" class="headerlink" title="5.7.1 开启动态分区参数设置"></a>5.7.1 开启动态分区参数设置</h4><ol>
<li>开启动态分区功能（默认为true，开启）</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>dynamic<span class="token punctuation">.</span><span class="token keyword">partition</span><span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<ol start="2">
<li>设置为非严格模式（动态分区的模式，默认strict，表示必须指定至少一个分区为静态分区,nonstrict模式表示允许所有的分区字段都可以使用动态分区。)</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>dynamic<span class="token punctuation">.</span><span class="token keyword">partition</span><span class="token punctuation">.</span>mode<span class="token operator">=</span>nonstrict<span class="token punctuation">;</span>
</code></pre>
<ol start="3">
<li>在所有执行MR的节点上，最大一共可以创建多少个动态分区。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>max<span class="token punctuation">.</span>dynamic<span class="token punctuation">.</span><span class="token keyword">partition</span><span class="token operator">=</span><span class="token number">1000</span><span class="token punctuation">;</span>
</code></pre>
<ol start="4">
<li>在每个执行MR的节点上，最大可以创建多少个动态分区。该参数需要根据实际的数据来设定。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>max<span class="token punctuation">.</span>dynamic<span class="token punctuation">.</span><span class="token keyword">partition</span><span class="token punctuation">.</span>pernode<span class="token operator">=</span><span class="token number">100</span><span class="token punctuation">;</span>
</code></pre>
<ol start="5">
<li>整个MR Job中，最大可以创建多少个HDFS文件。<br>在linux系统当中，每个linux用户最多可以开启1024个进程，每一个进程最多可以打开2048个文件，即持有2048个文件句柄，下面这个值越大，就可以打开文件句柄越大</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>max<span class="token punctuation">.</span>created<span class="token punctuation">.</span>files<span class="token operator">=</span><span class="token number">100000</span><span class="token punctuation">;</span>
</code></pre>
<ol start="6">
<li>当有空分区生成时，是否抛出异常。一般不需要设置。</li>
</ol>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>error<span class="token punctuation">.</span><span class="token keyword">on</span><span class="token punctuation">.</span>empty<span class="token punctuation">.</span><span class="token keyword">partition</span><span class="token operator">=</span><span class="token boolean">false</span><span class="token punctuation">;</span>
</code></pre>
<p>###5.8 并行执行</p>
<p>Hive会将一个查询转化成一个或者多个阶段。这样的阶段可以是MapReduce阶段、抽样阶段、合并阶段、limit阶段。或者Hive执行过程中可能需要的其他阶段。默认情况下，Hive一次只拿执行一个阶段。不过，某个特定的job可能包含众多的阶段，而这些阶段可能并非完全互相依赖的，也就是说有些阶段是可以并行执行的，这样可能使得整个job的执行时间缩短。不过，如果有更多的阶段可以并行执行，那么job可能就越快完成。</p>
<p>通过设置参数hive.exec.parallel值为true，就可以开启并发执行。不过，在共享集群中，需要注意下，如果job中并行阶段增多，那么集群利用率就会增加。</p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span><span class="token keyword">exec</span><span class="token punctuation">.</span>parallel<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>
<p>当然，得是在系统资源比较空菌的时候才有优势，否则，没资源，并行也起不来。</p>
<h3 id="5-9-严格模式"><a href="#5-9-严格模式" class="headerlink" title="5.9 严格模式"></a>5.9 严格模式</h3><p>Hive提供了一个严格模式，可以防止用户执行那些可能意向不到的不好的影响的查询。</p>
<p>通过设置属性hive.mapred.mode值为默认是非严格模式nonstrict。开启严格模式需要修改hive.mapred.mode值为strict，开启严格模式可以禁止3种类型的查询。</p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> hive<span class="token punctuation">.</span>mapred<span class="token punctuation">.</span>mode<span class="token operator">=</span>strict<span class="token punctuation">;</span>
<span class="token keyword">set</span> hive<span class="token punctuation">.</span>mapred<span class="token punctuation">.</span>mode<span class="token operator">=</span>nonstrict<span class="token punctuation">;</span>
</code></pre>
<ol>
<li>==对于分区表，在where语句中必须含有分区字段作为过滤条件来限制范围，否则不允许执行。==换句话说，就是用户不允许扫描所有分区。进行这个限制的原因是，通常分区表都拥有非常大的数据集，而且数据增加迅速。没有进行分区限制的查询可能会消耗令人不可接受的巨大资源来处理这个表。</li>
<li>==对于使用了order by语句的查询，要求必须使用limit语句。==因为order by为了执行排序过程会将所有的结果数据分发到同一个Reducer中进行处理，强制要求用户增加这个LIMIT语句可以防止Reducer额外执行很长一段时间。</li>
<li>==限制笛卡尔积的查询。==对关系型数据库非常了解的用户可能期望在执行JOIN查询的时候不使用ON语句而是使用where语句，这样关系数据库的执行优化器就可以高效地将WHERE语句转化成那个ON语句。不幸的是，Hive并不会执行这种优化，因此，如果表足够大，那么这个查询就会出现不可控的情况。</li>
</ol>
<h3 id="5-10-JVM重用"><a href="#5-10-JVM重用" class="headerlink" title="5.10 JVM重用"></a>5.10 JVM重用</h3><p>JVM重用是Hadoop调优参数的内容，其对Hive的性能具有非常大的影响，特别是对于很难避免小文件的场景或task特别多的场景，这类场景大多数执行时间都很短。</p>
<p>Hadoop的默认配置通常是使用派生JVM来执行map和Reduce任务的。这时JVM的启动过程可能会造成相当大的开销，尤其是执行的job包含有成百上千task任务的情况。JVM重用可以使得JVM实例在同一个job中重新使用N次。N的值可以在Hadoop的mapred-site.xml文件中进行配置。通常在10-20之间，具体多少需要根据具体业务场景测试得出。</p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapred<span class="token punctuation">.</span>job<span class="token punctuation">.</span>reuse<span class="token punctuation">.</span>jvm<span class="token punctuation">.</span>num<span class="token punctuation">.</span>tasks<span class="token operator">=</span><span class="token number">10</span><span class="token punctuation">;</span>
</code></pre>
<p>这个功能的缺点是，开启JVM重用将一直占用使用到的task插槽，以便进行重用，直到任务完成后才能释放。如果某个“不平衡的”job中有某几个reduce task执行的时间要比其他Reducetask消耗的时间多的多的话，那么保留的插槽就会一直空闲着却无法被其他的job使用，直到所有的task都结束了才会释放。</p>
<h3 id="5-11-推测执行"><a href="#5-11-推测执行" class="headerlink" title="5.11 推测执行"></a>5.11 推测执行</h3><p>在分布式集群环境下，因为程序Bug (包括Hadoop本身的bug)，负载不均衡或者资源分布不均等原因，会造成同一个作业的多个任务之间运行速度不一致，有些任务的运行速度可能明显慢于其他任务（比如一个作业的某个任务进度只有50%，而其他所有任务已经运行完毕)，则这些任务会拖慢作业的整体执行进度。为了避免这种情况发生，Hadoop采用了推测执行(Speculative Execution)机制，它根据一定的法则推测出”拖后腿”的任务，并为这样的任务启动一个备份任务，让该任务与原始任务同时处理同一份数据，并最终选用最先成功运行完成任务的计算结果作为最终结果。</p>
<p><strong>设置开启推测执行参数:</strong></p>
<pre class=" language-sql"><code class="language-sql"><span class="token keyword">set</span> mapred<span class="token punctuation">.</span>map<span class="token punctuation">.</span>tasks<span class="token punctuation">.</span>speculative<span class="token punctuation">.</span>execution<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
<span class="token keyword">set</span> mapred<span class="token punctuation">.</span>reduce<span class="token punctuation">.</span>tasks<span class="token punctuation">.</span>speculative<span class="token punctuation">.</span>execution<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
<span class="token keyword">set</span> hive<span class="token punctuation">.</span>mapred<span class="token punctuation">.</span>reduce<span class="token punctuation">.</span>tasks<span class="token punctuation">.</span>speculative<span class="token punctuation">.</span>execution<span class="token operator">=</span><span class="token boolean">true</span><span class="token punctuation">;</span>
</code></pre>

                
            </div>
            <hr/>

            

    <div class="reprint" id="reprint-statement">
        
            <div class="reprint__author">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-user">
                        文章作者:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="/about" rel="external nofollow noreferrer">Aunean</a>
                </span>
            </div>
            <div class="reprint__type">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-link">
                        文章链接:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="https://www.shiyiri.top/node/4.html">https://www.shiyiri.top/node/4.html</a>
                </span>
            </div>
            <div class="reprint__notice">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-copyright">
                        版权声明:
                    </i>
                </span>
                <span class="reprint-info">
                    本博客所有文章除特別声明外，均采用
                    <a href="https://creativecommons.org/licenses/by/4.0/deed.zh" rel="external nofollow noreferrer" target="_blank">CC BY 4.0</a>
                    许可协议。转载请注明来源
                    <a href="/about" target="_blank">Aunean</a>
                    !
                </span>
            </div>
        
    </div>

    <script async defer>
      document.addEventListener("copy", function (e) {
        let toastHTML = '<span>复制成功，请遵循本文的转载规则</span><button class="btn-flat toast-action" onclick="navToReprintStatement()" style="font-size: smaller">查看</a>';
        M.toast({html: toastHTML})
      });

      function navToReprintStatement() {
        $("html, body").animate({scrollTop: $("#reprint-statement").offset().top - 80}, 800);
      }
    </script>



            <div class="tag_share" style="display: block;">
                <div class="post-meta__tag-list" style="display: inline-block;">
                    
                        <div class="article-tag">
                            
                                <a href="/tags/Hive/">
                                    <span class="chip bg-color">Hive</span>
                                </a>
                            
                        </div>
                    
                </div>
                <div class="post_share" style="zoom: 80%; width: fit-content; display: inline-block; float: right; margin: -0.15rem 0;">
                    <link rel="stylesheet" type="text/css" href="/libs/share/css/share.min.css">
<div id="article-share">

    
    <div class="social-share" data-sites="twitter,facebook,google,qq,qzone,wechat,weibo,douban,linkedin" data-wechat-qrcode-helper="<p>微信扫一扫即可分享！</p>"></div>
    <script src="/libs/share/js/social-share.min.js"></script>
    

    

</div>

                </div>
            </div>
            
                <style>
    #reward {
        margin: 40px 0;
        text-align: center;
    }

    #reward .reward-link {
        font-size: 1.4rem;
        line-height: 38px;
    }

    #reward .btn-floating:hover {
        box-shadow: 0 6px 12px rgba(0, 0, 0, 0.2), 0 5px 15px rgba(0, 0, 0, 0.2);
    }

    #rewardModal {
        width: 320px;
        height: 350px;
    }

    #rewardModal .reward-title {
        margin: 15px auto;
        padding-bottom: 5px;
    }

    #rewardModal .modal-content {
        padding: 10px;
    }

    #rewardModal .close {
        position: absolute;
        right: 15px;
        top: 15px;
        color: rgba(0, 0, 0, 0.5);
        font-size: 1.3rem;
        line-height: 20px;
        cursor: pointer;
    }

    #rewardModal .close:hover {
        color: #ef5350;
        transform: scale(1.3);
        -moz-transform:scale(1.3);
        -webkit-transform:scale(1.3);
        -o-transform:scale(1.3);
    }

    #rewardModal .reward-tabs {
        margin: 0 auto;
        width: 210px;
    }

    .reward-tabs .tabs {
        height: 38px;
        margin: 10px auto;
        padding-left: 0;
    }

    .reward-content ul {
        padding-left: 0 !important;
    }

    .reward-tabs .tabs .tab {
        height: 38px;
        line-height: 38px;
    }

    .reward-tabs .tab a {
        color: #fff;
        background-color: #ccc;
    }

    .reward-tabs .tab a:hover {
        background-color: #ccc;
        color: #fff;
    }

    .reward-tabs .wechat-tab .active {
        color: #fff !important;
        background-color: #22AB38 !important;
    }

    .reward-tabs .alipay-tab .active {
        color: #fff !important;
        background-color: #019FE8 !important;
    }

    .reward-tabs .reward-img {
        width: 210px;
        height: 210px;
    }
</style>

<div id="reward">
    <a href="#rewardModal" class="reward-link modal-trigger btn-floating btn-medium waves-effect waves-light red">赏</a>

    <!-- Modal Structure -->
    <div id="rewardModal" class="modal">
        <div class="modal-content">
            <a class="close modal-close"><i class="fas fa-times"></i></a>
            <h4 class="reward-title">你的赏识是我前进的动力</h4>
            <div class="reward-content">
                <div class="reward-tabs">
                    <ul class="tabs row">
                        <li class="tab col s6 alipay-tab waves-effect waves-light"><a href="#alipay">支付宝</a></li>
                        <li class="tab col s6 wechat-tab waves-effect waves-light"><a href="#wechat">微 信</a></li>
                    </ul>
                    <div id="alipay">
                        <img src="/medias/reward/alipay.jpg" class="reward-img" alt="支付宝打赏二维码">
                    </div>
                    <div id="wechat">
                        <img src="/medias/reward/wechat.png" class="reward-img" alt="微信打赏二维码">
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script>
    $(function () {
        $('.tabs').tabs();
    });
</script>

            
        </div>
    </div>

    
        <link rel="stylesheet" href="/libs/gitalk/gitalk.css">
<link rel="stylesheet" href="/css/my-gitalk.css">

<div class="card gitalk-card" data-aos="fade-up">
    <div class="comment_headling" style="font-size: 20px; font-weight: 700; position: relative; padding-left: 20px; top: 15px; padding-bottom: 5px;">
        <i class="fas fa-comments fa-fw" aria-hidden="true"></i>
        <span>评论</span>
    </div>
    <div id="gitalk-container" class="card-content"></div>
</div>

<script src="/libs/gitalk/gitalk.min.js"></script>
<script>
    let gitalk = new Gitalk({
        clientID: '43918ae8f8557340e530',
        clientSecret: '016469b0d80284ccd95b27ea68909008a765d99e',
        repo: 'comment_by_blog',
        owner: 'Aunean-ls',
        admin: "Aunean-ls",
        id: '2021-05-31T12-18-00',
        distractionFreeMode: false  // Facebook-like distraction free mode
    });

    gitalk.render('gitalk-container');
</script>

    

    

    

    

    

    

    

    

<article id="prenext-posts" class="prev-next articles">
    <div class="row article-row">
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge left-badge text-color">
                <i class="fas fa-chevron-left"></i>&nbsp;上一篇</div>
            <div class="card">
                <a href="/node/103.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u012.webp" class="responsive-img" alt="scrapy存储json格式数据">
                        
                        <span class="card-title">scrapy存储json格式数据</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            四种scrapy存储为json格式方法
                        
                    </div>
                    <div class="publish-info">
                        <span class="publish-date">
                            <i class="far fa-clock fa-fw icon-date"></i>2021-06-06
                        </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/python/" class="post-category">
                                    python
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86/">
                        <span class="chip bg-color">数据采集</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge right-badge text-color">
                下一篇&nbsp;<i class="fas fa-chevron-right"></i>
            </div>
            <div class="card">
                <a href="/node/102.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u012.webp" class="responsive-img" alt="高性能爬虫">
                        
                        <span class="card-title">高性能爬虫</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            比较单线程、多线程、协程爬取速度
                        
                    </div>
                    <div class="publish-info">
                            <span class="publish-date">
                                <i class="far fa-clock fa-fw icon-date"></i>2021-05-31
                            </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/python/" class="post-category">
                                    python
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/%E6%95%B0%E6%8D%AE%E9%87%87%E9%9B%86/">
                        <span class="chip bg-color">数据采集</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
    </div>
</article>

</div>



<!-- 代码块功能依赖 -->
<script type="text/javascript" src="/libs/codeBlock/codeBlockFuction.js"></script>

<!-- 代码语言 -->

<script type="text/javascript" src="/libs/codeBlock/codeLang.js"></script>


<!-- 代码块复制 -->

<script type="text/javascript" src="/libs/codeBlock/codeCopy.js"></script>


<!-- 代码块收缩 -->

<script type="text/javascript" src="/libs/codeBlock/codeShrink.js"></script>


    </div>
    <div id="toc-aside" class="expanded col l3 hide-on-med-and-down">
        <div class="toc-widget card" style="background-color: white;">
            <div class="toc-title"><i class="far fa-list-alt"></i>&nbsp;&nbsp;目录</div>
            <div id="toc-content"></div>
        </div>
    </div>
</div>

<!-- TOC 悬浮按钮. -->

<div id="floating-toc-btn" class="hide-on-med-and-down">
    <a class="btn-floating btn-large bg-color">
        <i class="fas fa-list-ul"></i>
    </a>
</div>


<script src="/libs/tocbot/tocbot.min.js"></script>
<script>
    $(function () {
        tocbot.init({
            tocSelector: '#toc-content',
            contentSelector: '#articleContent',
            headingsOffset: -($(window).height() * 0.4 - 45),
            collapseDepth: Number('0'),
            headingSelector: 'h2, h3, h4, h5, h6'
        });

        // modify the toc link href to support Chinese.
        let i = 0;
        let tocHeading = 'toc-heading-';
        $('#toc-content a').each(function () {
            $(this).attr('href', '#' + tocHeading + (++i));
        });

        // modify the heading title id to support Chinese.
        i = 0;
        $('#articleContent').children('h2, h3, h4, h5, h6').each(function () {
            $(this).attr('id', tocHeading + (++i));
        });

        // Set scroll toc fixed.
        let tocHeight = parseInt($(window).height() * 0.4 - 64);
        let $tocWidget = $('.toc-widget');
        $(window).scroll(function () {
            let scroll = $(window).scrollTop();
            /* add post toc fixed. */
            if (scroll > tocHeight) {
                $tocWidget.addClass('toc-fixed');
            } else {
                $tocWidget.removeClass('toc-fixed');
            }
        });

        
        /* 修复文章卡片 div 的宽度. */
        let fixPostCardWidth = function (srcId, targetId) {
            let srcDiv = $('#' + srcId);
            if (srcDiv.length === 0) {
                return;
            }

            let w = srcDiv.width();
            if (w >= 450) {
                w = w + 21;
            } else if (w >= 350 && w < 450) {
                w = w + 18;
            } else if (w >= 300 && w < 350) {
                w = w + 16;
            } else {
                w = w + 14;
            }
            $('#' + targetId).width(w);
        };

        // 切换TOC目录展开收缩的相关操作.
        const expandedClass = 'expanded';
        let $tocAside = $('#toc-aside');
        let $mainContent = $('#main-content');
        $('#floating-toc-btn .btn-floating').click(function () {
            if ($tocAside.hasClass(expandedClass)) {
                $tocAside.removeClass(expandedClass).hide();
                $mainContent.removeClass('l9');
            } else {
                $tocAside.addClass(expandedClass).show();
                $mainContent.addClass('l9');
            }
            fixPostCardWidth('artDetail', 'prenext-posts');
        });
        
    });
</script>

    

</main>




    <footer class="page-footer bg-color">
    
        <link rel="stylesheet" href="/libs/aplayer/APlayer.min.css">
<style>
    .aplayer .aplayer-lrc p {
        
        display: none;
        
        font-size: 12px;
        font-weight: 700;
        line-height: 16px !important;
    }

    .aplayer .aplayer-lrc p.aplayer-lrc-current {
        
        display: none;
        
        font-size: 15px;
        color: #42b983;
    }

    
    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body {
        left: -66px !important;
    }

    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body:hover {
        left: 0px !important;
    }

    
</style>
<div class="">
    
    <div class="row">
        <meting-js class="col l8 offset-l2 m10 offset-m1 s12"
                   server="tencent"
                   type="playlist"
                   id="4628814494"
                   fixed='true'
                   autoplay='false'
                   theme='#42b983'
                   loop='all'
                   order='random'
                   preload='auto'
                   volume='0.7'
                   list-folded='true'
        >
        </meting-js>
    </div>
</div>

<script src="/libs/aplayer/APlayer.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/meting@2/dist/Meting.min.js"></script>

    
    <div class="container row center-align" style="margin-bottom: 15px !important;">
        <div class="col s12 m8 l8 copy-right">
            Copyright&nbsp;&copy;
            
                <span id="year">2021</span>
            
            <span id="year">2021</span>
            <a href="/about" target="_blank">Aunean</a>
            |&nbsp;Powered by&nbsp;<a href="https://hexo.io/" target="_blank">Hexo</a>
            |&nbsp;Theme&nbsp;<a href="https://github.com/blinkfox/hexo-theme-matery" target="_blank">Matery</a>
            <br>
            
            &nbsp;<i class="fas fa-chart-area"></i>&nbsp;站点总字数:&nbsp;<span
                class="white-color">64.8k</span>&nbsp;字
            
            
            
            
            
            
            <span id="busuanzi_container_site_pv">
                |&nbsp;<i class="far fa-eye"></i>&nbsp;总访问量:&nbsp;<span id="busuanzi_value_site_pv"
                    class="white-color"></span>&nbsp;次
            </span>
            
            
            <span id="busuanzi_container_site_uv">
                |&nbsp;<i class="fas fa-users"></i>&nbsp;总访问人数:&nbsp;<span id="busuanzi_value_site_uv"
                    class="white-color"></span>&nbsp;人
            </span>
            
            <br>
            
            <span id="sitetime">载入运行时间...</span>
            <script>
                function siteTime() {
                    var seconds = 1000;
                    var minutes = seconds * 60;
                    var hours = minutes * 60;
                    var days = hours * 24;
                    var years = days * 365;
                    var today = new Date();
                    var startYear = "2021";
                    var startMonth = "4";
                    var startDate = "11";
                    var startHour = "0";
                    var startMinute = "0";
                    var startSecond = "0";
                    var todayYear = today.getFullYear();
                    var todayMonth = today.getMonth() + 1;
                    var todayDate = today.getDate();
                    var todayHour = today.getHours();
                    var todayMinute = today.getMinutes();
                    var todaySecond = today.getSeconds();
                    var t1 = Date.UTC(startYear, startMonth, startDate, startHour, startMinute, startSecond);
                    var t2 = Date.UTC(todayYear, todayMonth, todayDate, todayHour, todayMinute, todaySecond);
                    var diff = t2 - t1;
                    var diffYears = Math.floor(diff / years);
                    var diffDays = Math.floor((diff / days) - diffYears * 365);
                    var diffHours = Math.floor((diff - (diffYears * 365 + diffDays) * days) / hours);
                    var diffMinutes = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours) /
                        minutes);
                    var diffSeconds = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours -
                        diffMinutes * minutes) / seconds);
                    if (startYear == todayYear) {
                        document.getElementById("year").innerHTML = todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffDays + " 天 " + diffHours +
                            " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    } else {
                        document.getElementById("year").innerHTML = startYear + " - " + todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffYears + " 年 " + diffDays +
                            " 天 " + diffHours + " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    }
                }
                setInterval(siteTime, 1000);
            </script>
            
            <br>
            
        </div>
        <div class="col s12 m4 l4 social-link social-statis">
    <a href="https://github.com/Aunean-ls" class="tooltipped" target="_blank" data-tooltip="访问我的GitHub" data-position="top" data-delay="50">
        <i class="fab fa-github"></i>
    </a>



    <a href="mailto:1453357375@qq.com" class="tooltipped" target="_blank" data-tooltip="邮件联系我" data-position="top" data-delay="50">
        <i class="fas fa-envelope-open"></i>
    </a>







    <a href="tencent://AddContact/?fromId=50&fromSubId=1&subcmd=all&uin=1453357375" class="tooltipped" target="_blank" data-tooltip="QQ联系我: 1453357375" data-position="top" data-delay="50">
        <i class="fab fa-qq"></i>
    </a>







    <a href="/atom.xml" class="tooltipped" target="_blank" data-tooltip="RSS 订阅" data-position="top" data-delay="50">
        <i class="fas fa-rss"></i>
    </a>

</div>
    </div>
</footer>

<div class="progress-bar"></div>


    <!-- 搜索遮罩框 -->
<div id="searchModal" class="modal">
    <div class="modal-content">
        <div class="search-header">
            <span class="title"><i class="fas fa-search"></i>&nbsp;&nbsp;搜索</span>
            <input type="search" id="searchInput" name="s" placeholder="请输入搜索的关键字"
                   class="search-input">
        </div>
        <div id="searchResult"></div>
    </div>
</div>

<script type="text/javascript">
$(function () {
    var searchFunc = function (path, search_id, content_id) {
        'use strict';
        $.ajax({
            url: path,
            dataType: "xml",
            success: function (xmlResponse) {
                // get the contents from search data
                var datas = $("entry", xmlResponse).map(function () {
                    return {
                        title: $("title", this).text(),
                        content: $("content", this).text(),
                        url: $("url", this).text()
                    };
                }).get();
                var $input = document.getElementById(search_id);
                var $resultContent = document.getElementById(content_id);
                $input.addEventListener('input', function () {
                    var str = '<ul class=\"search-result-list\">';
                    var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                    $resultContent.innerHTML = "";
                    if (this.value.trim().length <= 0) {
                        return;
                    }
                    // perform local searching
                    datas.forEach(function (data) {
                        var isMatch = true;
                        var data_title = data.title.trim().toLowerCase();
                        var data_content = data.content.trim().replace(/<[^>]+>/g, "").toLowerCase();
                        var data_url = data.url;
                        data_url = data_url.indexOf('/') === 0 ? data.url : '/' + data_url;
                        var index_title = -1;
                        var index_content = -1;
                        var first_occur = -1;
                        // only match artiles with not empty titles and contents
                        if (data_title !== '' && data_content !== '') {
                            keywords.forEach(function (keyword, i) {
                                index_title = data_title.indexOf(keyword);
                                index_content = data_content.indexOf(keyword);
                                if (index_title < 0 && index_content < 0) {
                                    isMatch = false;
                                } else {
                                    if (index_content < 0) {
                                        index_content = 0;
                                    }
                                    if (i === 0) {
                                        first_occur = index_content;
                                    }
                                }
                            });
                        }
                        // show search results
                        if (isMatch) {
                            str += "<li><a href='" + data_url + "' class='search-result-title'>" + data_title + "</a>";
                            var content = data.content.trim().replace(/<[^>]+>/g, "");
                            if (first_occur >= 0) {
                                // cut out 100 characters
                                var start = first_occur - 20;
                                var end = first_occur + 80;
                                if (start < 0) {
                                    start = 0;
                                }
                                if (start === 0) {
                                    end = 100;
                                }
                                if (end > content.length) {
                                    end = content.length;
                                }
                                var match_content = content.substr(start, end);
                                // highlight all keywords
                                keywords.forEach(function (keyword) {
                                    var regS = new RegExp(keyword, "gi");
                                    match_content = match_content.replace(regS, "<em class=\"search-keyword\">" + keyword + "</em>");
                                });

                                str += "<p class=\"search-result\">" + match_content + "...</p>"
                            }
                            str += "</li>";
                        }
                    });
                    str += "</ul>";
                    $resultContent.innerHTML = str;
                });
            }
        });
    };

    searchFunc('/search.xml', 'searchInput', 'searchResult');
});
</script>

    <!-- 回到顶部按钮 -->
<div id="backTop" class="top-scroll">
    <a class="btn-floating btn-large waves-effect waves-light" href="#!">
        <i class="fas fa-arrow-up"></i>
    </a>
</div>


    <script src="/libs/materialize/materialize.min.js"></script>
    <script src="/libs/masonry/masonry.pkgd.min.js"></script>
    <script src="/libs/aos/aos.js"></script>
    <script src="/libs/scrollprogress/scrollProgress.min.js"></script>
    <script src="/libs/lightGallery/js/lightgallery-all.min.js"></script>
    <script src="/js/matery.js"></script>

    <script type="text/javascript">
        var OriginTitile=document.title,st;
        document.addEventListener("visibilitychange",function(){
            document.hidden?(document.title="ヽ(●-`Д´-)ノ页面丢失了",clearTimeout(st)):(document.title="(Ő∀Ő3)ノ又好了哦！",st=setTimeout(function(){document.title=OriginTitile},3e3))
        })
    </script>


    <script type="text/javascript">
    //只在桌面版网页启用特效
        var windowWidth = $(window).width();
        if (windowWidth > 768) {
            document.write('<script type="text/javascript" src="/js/sakura.js"><\/script>');
        }
    </script>


    <!-- Baidu Analytics -->

    <!-- Baidu Push -->

<script>
    (function () {
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https') {
            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
        } else {
            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
        }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>

    
    <script src="/libs/others/clicklove.js" async="async"></script>
    
    
    <script async src="/libs/others/busuanzi.pure.mini.js"></script>
    

    

    

	
    
    <script type="text/javascript" color="0,0,255"
        pointColor="0,0,255" opacity='0.7'
        zIndex="-1" count="99"
        src="/libs/background/canvas-nest.js"></script>
    

    
    
    <script type="text/javascript" size="150" alpha='0.6'
        zIndex="-1" src="/libs/background/ribbon-refresh.min.js" async="async"></script>
    

    
    <script type="text/javascript" src="/libs/background/ribbon-dynamic.js" async="async"></script>
    

    
    <script src="/libs/instantpage/instantpage.js" type="module"></script>
    


</body>

</html>
